package com.gxljc.bear.crawler.util;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;

import com.gxljc.commons.util.sql.BaseDao;
import com.gxljc.commons.util.sql.BaseDaoFactory;
import com.gxljc.bear.crawler.base.DolphinCrawlerConf;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;

public class SiteMappingUtil {

	public void siteMapping() throws IOException {
		String filename = "site-mapping.txt";
		List<String> domaninList = extractDomains(filename);// <url>\t<sitename>\t<domain>

		Configuration conf = DolphinCrawlerConf.getInstance();
		BaseDao baseDao = BaseDaoFactory.getDaoBaseInstance(conf
				.get(DolphinCrawlerConsts.PARAM_MYSQL_URL));
		int duplicateCount = 0;
		int insertCount = 0;
		for (String s : domaninList) {
			String[] split = s.split("\t");
			String url = split[0];
			String sitename = split[1];
			String domain = split[2];
			String sql = String.format(
					"select web_name from %s where web_name=?",
					"t_mobile_web_link_mapping");
			String result = null;
			try {
				result = baseDao.queryColumn(sql, sitename);
			} catch (SQLException e) {
				e.printStackTrace();
			}
			if (result != null) {
				System.out.println(sitename + "\t" + result);
				duplicateCount++;
			} else {
				System.out.println("insert:" + "\t" + s);
				String insertSql = String
						.format("insert into t_mobile_web_link_mapping"
								+ " (web_name,web_host,domain,pv,source) values(?,?,?,?,?)");
				try {
					baseDao.batchExecute(insertSql, sitename, url, domain, 0,
							"yuqing");
					insertCount++;
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
		System.out.println(domaninList.size() + "\t" + duplicateCount + "\t"
				+ insertCount);
	}

	private List<String> extractDomains(String filename) throws IOException {
		List<String> domaninList = new ArrayList<String>();
		List<String> lines = FileUtils.readLines(new File(filename));
		Pattern pattern = Pattern
				.compile("\\.?(\\w{1,})\\.(com|cn|net|org|hk)");
		for (String line : lines) {
			String[] split = line.split("\t");
			if (split.length != 2) {
				System.err.println(line);
				continue;
			}
			String url = split[0];
			String sitename = split[1];
			if (sitename.contains(".")) {
				continue;
			}
			Matcher matcher = pattern.matcher(url);
			String domain = null;
			if (matcher.find()) {
				domain = matcher.group(1);
			} else {
				String[] tokens = url.split("\\.");
				if (tokens.length < 2) {
					continue;
				}
				domain = tokens[tokens.length - 2];
			}
			domaninList.add(line + "\t" + domain);
		}
		return domaninList;
	}
}
